Find cars on an image
In [1]:
import numpy as np
import pickle
import cv2
import glob
import os
import time
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import pandas as pd
from moviepy.editor import VideoFileClip
from skimage.feature import hog
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from scipy.ndimage.measurements import label
Data collection
In [2]:
car_images = glob.glob('./training_data/vehicles/*/*/*.png')
noncar_images = glob.glob('./training_data/non_vehicles/*/*/*.png')
Set hog parameters, result of hog parameter exploration step
In [3]:
hog_params = {}
hog_params['cspace'] = 'YUV'
hog_params['orient'] = 10
hog_params['pix_per_cell'] = 16
hog_params['cell_per_block'] = 2
hog_params['hog_channel'] = "ALL"
Train using SVC after getting hog features
In [4]:
from vehicle_detection import classifier
print('Training with hog features...........')
g_svc, hog_svc_data = classifier.train_hog_features(car_images, noncar_images, hog_params)
print('Training done...........')
Training with hog features...........
43.4 Seconds to extract HOG features...
Using: 10 orientations 16 pixels per cell and 2 cells per block
Feature vector length: 1080
1.01 Seconds to train SVC...
Test Accuracy of SVC =  0.9825
My SVC predicts:  [ 1.  1.  0.  1.  1.  1.  1.  0.  0.  0.]
For these 10 labels:  [ 1.  1.  0.  1.  1.  1.  1.  0.  0.  0.]
0.01755 Seconds to predict 10 labels with SVC
Training done...........
In [5]:
from vehicle_detection import sliding_window
Set parameters for window, a set of choice and try on an image
In [6]:
g_ystart = 400
g_ystop = 650
g_scale = 1.5

g_colorspace = hog_params['cspace']
g_hog_channel = hog_params['hog_channel']
g_orient = hog_params['orient']
g_pix_per_cell = hog_params['pix_per_cell']
g_cell_per_block = hog_params['cell_per_block']
In [7]:
img = cv2.imread('./test_images/test1.jpg')
img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
test_image = img.copy()

display(img.shape)

print('Applying sliding widnow search...........')
car_windows = sliding_window.detect_cars(img, g_ystart, g_ystop, g_scale, g_svc, g_colorspace, g_hog_channel, 
                          g_orient, g_pix_per_cell, g_cell_per_block)
display(car_windows)
print(len(car_windows), 'rectangles found in image')
car_detected_img = sliding_window.draw_boxes(img, car_windows)
plt.figure(figsize=(10,10))
plt.imshow(car_detected_img)
plt.show()
(720, 1280, 3)
Applying sliding widnow search...........
[((48, 400), (144, 496)),
 ((144, 448), (240, 544)),
 ((192, 400), (288, 496)),
 ((816, 400), (912, 496)),
 ((864, 400), (960, 496)),
 ((1056, 400), (1152, 496)),
 ((1104, 400), (1200, 496)),
 ((1152, 400), (1248, 496))]
8 rectangles found in image
Function to perform an exhaustive search, I have chosen the coorodinates and scaling by manual observation of areas that will have cars
In [8]:
def car_location_exploration(img, ystart, ystop, img_scale):
    car_windows = []
    for start in ystart:
            for stop in ystop:
                for scale in img_scale:
                    g_ystart = start
                    g_ystop = stop
                    g_scale = scale
                    
                    car_windows.append(sliding_window.detect_cars(img, g_ystart, g_ystop, g_scale, g_svc, g_colorspace, g_hog_channel, 
                              g_orient, g_pix_per_cell, g_cell_per_block))
    return car_windows
In [9]:
ystart = [300,350,400, 425]
ystop = [500, 550, 600, 650]
img_scale = [1, 1.5, 2, 2.5]

print('Locating cars from different locations...........')
car_windows = car_location_exploration(test_image, ystart, ystop, img_scale)
Locating cars from different locations...........
Draw detected rectangles
In [10]:
# apparently this is the best way to flatten a list of lists
car_positions = [item for sublist in car_windows for item in sublist] 
test_img_cars = sliding_window.draw_boxes(test_image, car_positions)
plt.figure(figsize=(10,10))
plt.imshow(test_img_cars)
plt.show()
Test heat maps
In [11]:
# Test out the heatmap
heatmap_img = np.zeros_like(test_image[:,:,0])
heatmap_img = sliding_window.add_heat(heatmap_img, car_positions)
plt.figure(figsize=(10,10))
plt.imshow(heatmap_img, cmap='hot')
plt.show()
Apply threshold to heat map, chosen threshold is 5 based on trial and error
In [12]:
heatmap_img = sliding_window.apply_threshold(heatmap_img, 5)
plt.figure(figsize=(10,10))
plt.imshow(heatmap_img, cmap='hot')
plt.show()
Apply scipy labels to heatmaps
In [13]:
labels = label(heatmap_img)
plt.figure(figsize=(10,10))
plt.imshow(labels[0], cmap='gray')
print(labels[1], 'cars found')
plt.show()
3 cars found
Draw bounding boxes for labels
In [14]:
# Draw bounding boxes on a copy of the image
draw_img = sliding_window.draw_labeled_bboxes(np.copy(test_image), labels)
# Display the image
plt.figure(figsize=(10,10))
plt.imshow(draw_img)
plt.show()
In [15]:
def detect_car_pipeline(test_image, plot=False, sw_params = None, car_pos =  None):

    if plot:
        f, (axis0, axis1, axis2, axis3, axis4) = plt.subplots(5, 1, figsize=(20, 30))
        axis0.imshow(test_image)
        axis0.set_title('Original image', fontsize=14)
    
    if sw_params and plot:
        car_windows = car_location_exploration_fast(test_image, sw_params)
    elif plot:
        car_windows = car_location_exploration(test_image, g_ystart, g_ystop, g_img_scale)
    else:
        car_windows = car_location_exploration_fast(test_image, g_sliding_window_params)
    car_positions = [item for sublist in car_windows for item in sublist] 
    test_img_cars = sliding_window.draw_boxes(test_image, car_positions)
    if plot:
        axis1.imshow(test_img_cars)
        axis1.set_title('Sliding window results', fontsize=14)
    
    # Test out the heatmap
    heatmap_img = np.zeros_like(test_image[:,:,0])
    if not car_pos:
        heatmap_img = sliding_window.add_heat(heatmap_img, car_positions)
    else:
        for positions in car_pos.recent_car_positions:
            heatmap_img = sliding_window.add_heat(heatmap_img, positions)
    if plot:
        axis2.imshow(heatmap_img, cmap='hot')
        axis2.set_title('Add heatmap', fontsize=14)
    if not car_pos:
        thresholded_img = sliding_window.apply_threshold(heatmap_img, g_threshold)
    else:
        thresholded_img = sliding_window.apply_threshold(heatmap_img, g_threshold)
    if plot:
        axis3.imshow(thresholded_img, cmap='gray')
        axis3.set_title('Thresholded image', fontsize=14)
    
    labels = label(thresholded_img)
    
    # Draw bounding boxes on a copy of the image
    draw_img = sliding_window.draw_labeled_bboxes(np.copy(test_image), labels)   
    # Display the image
    if plot:
        axis4.imshow(draw_img)
        axis4.set_title('Cars detected', fontsize=14)

    if plot:
        return plt.figure()
    else:
        return draw_img
In [16]:
g_ystart = [350, 400, 450]
g_ystop = [500, 550, 600, 650]
g_img_scale = [1, 1.5, 2, 2.5, 3]

g_threshold = 5
In [17]:
test_images = glob.glob('./test_images/test*.jpg')
out_dir = ('./output_images')

for idx, img in enumerate(test_images):
    image_name = img.split('/')[1].split('\\')[1]
    print(image_name)
    image = cv2.imread(img)
    image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
    car_image = detect_car_pipeline(image, plot=True)

    if not os.path.exists(out_dir):
        os.makedirs(out_dir)
    plt.show(car_image)
test1.jpg
<matplotlib.figure.Figure at 0x1fae23bec18>
test2.jpg
<matplotlib.figure.Figure at 0x1fae02f35c0>
test3.jpg
<matplotlib.figure.Figure at 0x1fae3f07748>
test4.jpg
<matplotlib.figure.Figure at 0x1fae3f3e978>
test5.jpg
<matplotlib.figure.Figure at 0x1fada174470>
test6.jpg
<matplotlib.figure.Figure at 0x1fae3e6a5c0>

Try the same on project video to see how it performs

Faster Sliding windows detection

It is clear that computation cost is very higher to search across many windows with many scaling. As explained in the lecture, it is better to identify the correct car locations in the image and have minimum scaling in cars infront and maximum scaling for cars closer to the camera.

Chosen extreme points for windows:

ystart: 400
ystop: 650

Add a list of parameters in the order of ystart, ystop and scaling value. Smaller windows are scaled lesser and Bigger windows are scaled higher.

In [19]:
g_sliding_window_params = [[400,450,1.5],[425,475,1.5], [400,475,1.5], [425,500,1.5],[400,500,1.5],[400,525,1.5],[400,600,1.5],[450,650,1.5]]
g_threshold = 1
In [20]:
def car_location_exploration_fast(img, sw_params):
    car_windows = []
    for params in sw_params:
        ystart = params[0]
        ystop = params[1]
        scale = params[2]
        print(ystart, ystop, scale)
            
        #Detect cars
        car_windows.append(sliding_window.detect_cars(img, ystart, ystop, scale, g_svc, g_colorspace, g_hog_channel, 
                  g_orient, g_pix_per_cell, g_cell_per_block))
    print(car_windows)
    return car_windows
In [21]:
test_images = glob.glob('./test_images/test*.jpg')
out_dir = ('./output_images/sw_fast')

for idx, img in enumerate(test_images):
    image_name = img.split('/')[1].split('\\')[1]
    print(image_name)
    image = cv2.imread(img)
    image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
    car_image = detect_car_pipeline(image, True, g_sliding_window_params)

    if not os.path.exists(out_dir):
        os.makedirs(out_dir)
    plt.show(car_image)
test1.jpg
400 450 1.5
425 475 1.5
400 475 1.5
425 500 1.5
400 500 1.5
400 525 1.5
400 600 1.5
450 650 1.5
[[], [], [], [], [((48, 400), (144, 496)), ((192, 400), (288, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496)), ((1152, 400), (1248, 496))], [((48, 400), (144, 496)), ((192, 400), (288, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496)), ((1152, 400), (1248, 496))], [((48, 400), (144, 496)), ((144, 448), (240, 544)), ((192, 400), (288, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496)), ((1152, 400), (1248, 496))], [((144, 450), (240, 546))]]
<matplotlib.figure.Figure at 0x1fae3aa1be0>
test2.jpg
400 450 1.5
425 475 1.5
400 475 1.5
425 500 1.5
400 500 1.5
400 525 1.5
400 600 1.5
450 650 1.5
[[], [], [], [], [], [], [], []]
<matplotlib.figure.Figure at 0x1fae3f3a940>
test3.jpg
400 450 1.5
425 475 1.5
400 475 1.5
425 500 1.5
400 500 1.5
400 525 1.5
400 600 1.5
450 650 1.5
[[], [], [], [], [((864, 400), (960, 496))], [((864, 400), (960, 496))], [((864, 400), (960, 496))], []]
<matplotlib.figure.Figure at 0x1fae4034b70>
test4.jpg
400 450 1.5
425 475 1.5
400 475 1.5
425 500 1.5
400 500 1.5
400 525 1.5
400 600 1.5
450 650 1.5
[[], [], [], [], [((96, 400), (192, 496)), ((192, 400), (288, 496)), ((240, 400), (336, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496)), ((1152, 400), (1248, 496))], [((192, 400), (288, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496)), ((1152, 400), (1248, 496))], [((48, 448), (144, 544)), ((96, 448), (192, 544)), ((192, 400), (288, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496)), ((1152, 400), (1248, 496))], [((48, 450), (144, 546)), ((96, 450), (192, 546))]]
<matplotlib.figure.Figure at 0x1fae136d668>
test5.jpg
400 450 1.5
425 475 1.5
400 475 1.5
425 500 1.5
400 500 1.5
400 525 1.5
400 600 1.5
450 650 1.5
[[], [], [], [], [((0, 400), (96, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1104, 400), (1200, 496))], [((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1104, 400), (1200, 496))], [((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1104, 400), (1200, 496))], []]
<matplotlib.figure.Figure at 0x1fae40cdf60>
test6.jpg
400 450 1.5
425 475 1.5
400 475 1.5
425 500 1.5
400 500 1.5
400 525 1.5
400 600 1.5
450 650 1.5
[[], [], [], [], [((96, 400), (192, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1008, 400), (1104, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496))], [((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1008, 400), (1104, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496))], [((0, 400), (96, 496)), ((816, 400), (912, 496)), ((864, 400), (960, 496)), ((1008, 400), (1104, 496)), ((1056, 400), (1152, 496)), ((1104, 400), (1200, 496))], []]
<matplotlib.figure.Figure at 0x1fae485e668>
In [ ]:
g_sliding_window_params = [[400,450,1.5],[425,475,1.5], [400,475,1.5], [425,500,1.5],[400,500,1.5],[400,525,1.5],[400,600,1.5],[450,650,1.5]]
g_threshold = 1
from moviepy.editor import VideoFileClip
test_out_file = 'project_video_out_lesswindows.mp4'
clip_test = VideoFileClip('project_video.mp4')
clip_test_out = clip_test.fl_image(detect_car_pipeline)
%time clip_test_out.write_videofile(test_out_file, audio=False)

Get all frames from video for working with false positives

In [ ]:
import cv2
print(cv2.__version__)
vidcap = cv2.VideoCapture('project_video.mp4')
success,image = vidcap.read()
count = 0
success = True
while success:
    success, image = vidcap.read()
    cv2.imwrite("project_video_images/frame%d.jpg" % count, image)     # save frame as JPEG file
    count += 1

First few images give me a false positive, process those!

In [ ]:
proj_images = [glob.glob('./project_video_images/frame*.jpg')]
proj_images.sort()
count = 0
for idx, img in enumerate(proj_images[0]):
    if count < 25:
        image = cv2.imread(img)
        image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
        car_image = detect_car_pipeline(image, True, g_sliding_window_params)
        plt.show(car_image)
    else:
        break